{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.001, 0.01, 0.1, 1, 10, 100],\n",
       " 'reg_lambda': [0.001, 0.01, 0.1, 1, 10, 100]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 正则参数\n",
    "reg_alpha = [pow(10,i) for i in range(-3,3,1)]\n",
    "reg_lambda = [pow(10,i) for i in range(-3,3,1)]\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58432, std: 0.00405, params: {'reg_alpha': 0.001, 'reg_lambda': 0.001},\n",
       "  mean: -0.58470, std: 0.00374, params: {'reg_alpha': 0.001, 'reg_lambda': 0.01},\n",
       "  mean: -0.58442, std: 0.00391, params: {'reg_alpha': 0.001, 'reg_lambda': 0.1},\n",
       "  mean: -0.58395, std: 0.00382, params: {'reg_alpha': 0.001, 'reg_lambda': 1},\n",
       "  mean: -0.58443, std: 0.00327, params: {'reg_alpha': 0.001, 'reg_lambda': 10},\n",
       "  mean: -0.58856, std: 0.00328, params: {'reg_alpha': 0.001, 'reg_lambda': 100},\n",
       "  mean: -0.58458, std: 0.00386, params: {'reg_alpha': 0.01, 'reg_lambda': 0.001},\n",
       "  mean: -0.58423, std: 0.00360, params: {'reg_alpha': 0.01, 'reg_lambda': 0.01},\n",
       "  mean: -0.58413, std: 0.00383, params: {'reg_alpha': 0.01, 'reg_lambda': 0.1},\n",
       "  mean: -0.58430, std: 0.00382, params: {'reg_alpha': 0.01, 'reg_lambda': 1},\n",
       "  mean: -0.58475, std: 0.00338, params: {'reg_alpha': 0.01, 'reg_lambda': 10},\n",
       "  mean: -0.58855, std: 0.00325, params: {'reg_alpha': 0.01, 'reg_lambda': 100},\n",
       "  mean: -0.58436, std: 0.00360, params: {'reg_alpha': 0.1, 'reg_lambda': 0.001},\n",
       "  mean: -0.58420, std: 0.00389, params: {'reg_alpha': 0.1, 'reg_lambda': 0.01},\n",
       "  mean: -0.58435, std: 0.00375, params: {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58420, std: 0.00393, params: {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  mean: -0.58451, std: 0.00327, params: {'reg_alpha': 0.1, 'reg_lambda': 10},\n",
       "  mean: -0.58903, std: 0.00321, params: {'reg_alpha': 0.1, 'reg_lambda': 100},\n",
       "  mean: -0.58384, std: 0.00367, params: {'reg_alpha': 1, 'reg_lambda': 0.001},\n",
       "  mean: -0.58357, std: 0.00334, params: {'reg_alpha': 1, 'reg_lambda': 0.01},\n",
       "  mean: -0.58311, std: 0.00326, params: {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58370, std: 0.00341, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.58479, std: 0.00363, params: {'reg_alpha': 1, 'reg_lambda': 10},\n",
       "  mean: -0.58902, std: 0.00326, params: {'reg_alpha': 1, 'reg_lambda': 100},\n",
       "  mean: -0.58795, std: 0.00315, params: {'reg_alpha': 10, 'reg_lambda': 0.001},\n",
       "  mean: -0.58797, std: 0.00307, params: {'reg_alpha': 10, 'reg_lambda': 0.01},\n",
       "  mean: -0.58783, std: 0.00318, params: {'reg_alpha': 10, 'reg_lambda': 0.1},\n",
       "  mean: -0.58807, std: 0.00297, params: {'reg_alpha': 10, 'reg_lambda': 1},\n",
       "  mean: -0.58874, std: 0.00288, params: {'reg_alpha': 10, 'reg_lambda': 10},\n",
       "  mean: -0.59240, std: 0.00302, params: {'reg_alpha': 10, 'reg_lambda': 100},\n",
       "  mean: -0.62187, std: 0.00312, params: {'reg_alpha': 100, 'reg_lambda': 0.001},\n",
       "  mean: -0.62184, std: 0.00307, params: {'reg_alpha': 100, 'reg_lambda': 0.01},\n",
       "  mean: -0.62186, std: 0.00308, params: {'reg_alpha': 100, 'reg_lambda': 0.1},\n",
       "  mean: -0.62183, std: 0.00320, params: {'reg_alpha': 100, 'reg_lambda': 1},\n",
       "  mean: -0.62183, std: 0.00308, params: {'reg_alpha': 100, 'reg_lambda': 10},\n",
       "  mean: -0.62288, std: 0.00321, params: {'reg_alpha': 100, 'reg_lambda': 100}],\n",
       " {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       " -0.5831126204138123)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=267,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([150.6238656 , 146.94912939, 144.75799646, 128.57803321,\n",
       "        120.63372846, 122.70400238, 120.91174097, 121.45325346,\n",
       "        120.48500137, 121.41106162, 123.99452572, 122.91807747,\n",
       "        122.78056355, 121.89249158, 125.42329884, 123.26945553,\n",
       "        122.08527594, 123.94196877, 123.40150113, 121.67266831,\n",
       "        124.08403654, 122.10758758, 122.15317936, 122.4684432 ,\n",
       "        124.34694624, 123.2065125 , 123.02132006, 122.49946008,\n",
       "        123.26337104, 123.67287064,  99.05490336, 101.1958993 ,\n",
       "         99.60997944,  98.74098363, 100.18406901,  77.291505  ]),\n",
       " 'std_fit_time': array([ 1.07305597,  3.58117668,  1.40840385,  8.20209637,  2.22684425,\n",
       "         4.02247783,  2.96995541,  2.78934963,  2.98332165,  2.70270923,\n",
       "         1.61562046,  3.28941346,  2.6158334 ,  3.18740422,  3.86492177,\n",
       "         4.68652214,  1.73359089,  4.01084881,  2.16884457,  2.12451719,\n",
       "         1.87572896,  2.69880536,  3.01026891,  1.75761502,  1.91424305,\n",
       "         2.76877325,  1.45447899,  2.87264538,  0.89627762,  3.62247928,\n",
       "         2.68342185,  3.28139999,  3.19084057,  2.50672312,  1.68766831,\n",
       "        12.82889206]),\n",
       " 'mean_score_time': array([0.67845149, 0.54427104, 0.48541527, 0.57838936, 0.56474795,\n",
       "        0.53796382, 0.5183228 , 0.55934801, 0.47470379, 0.4949769 ,\n",
       "        0.5131515 , 0.4915874 , 0.52358274, 0.51963401, 0.48209133,\n",
       "        0.48135514, 0.48940992, 0.57319255, 0.52470469, 0.48094177,\n",
       "        0.48523011, 0.48398061, 0.48822923, 0.50888596, 0.5821744 ,\n",
       "        0.493261  , 0.54073834, 0.5035604 , 0.50778513, 0.52337008,\n",
       "        0.36497855, 0.3643909 , 0.381634  , 0.35735569, 0.34761248,\n",
       "        0.26391106]),\n",
       " 'std_score_time': array([0.23081516, 0.06868307, 0.01762472, 0.06847632, 0.08385019,\n",
       "        0.08977229, 0.08107712, 0.09372101, 0.00960895, 0.04579621,\n",
       "        0.03235   , 0.01084604, 0.08025965, 0.05683639, 0.00815733,\n",
       "        0.00971828, 0.01384397, 0.10827926, 0.07573538, 0.00209121,\n",
       "        0.00706892, 0.00645443, 0.01061087, 0.04252134, 0.08705867,\n",
       "        0.00470354, 0.05753942, 0.01558746, 0.02964787, 0.02893205,\n",
       "        0.0109118 , 0.00883891, 0.05455673, 0.00783475, 0.0365952 ,\n",
       "        0.00608288]),\n",
       " 'param_reg_alpha': masked_array(data=[0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.01, 0.01,\n",
       "                    0.01, 0.01, 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1,\n",
       "                    1, 1, 1, 1, 1, 1, 10, 10, 10, 10, 10, 10, 100, 100,\n",
       "                    100, 100, 100, 100],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.001, 0.01, 0.1, 1, 10, 100, 0.001, 0.01, 0.1, 1, 10,\n",
       "                    100, 0.001, 0.01, 0.1, 1, 10, 100, 0.001, 0.01, 0.1, 1,\n",
       "                    10, 100, 0.001, 0.01, 0.1, 1, 10, 100, 0.001, 0.01,\n",
       "                    0.1, 1, 10, 100],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 0.001, 'reg_lambda': 0.001},\n",
       "  {'reg_alpha': 0.001, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 0.001, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 0.001, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 0.001, 'reg_lambda': 10},\n",
       "  {'reg_alpha': 0.001, 'reg_lambda': 100},\n",
       "  {'reg_alpha': 0.01, 'reg_lambda': 0.001},\n",
       "  {'reg_alpha': 0.01, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 0.01, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 0.01, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 0.01, 'reg_lambda': 10},\n",
       "  {'reg_alpha': 0.01, 'reg_lambda': 100},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 0.001},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 10},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 100},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.001},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 10},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 100},\n",
       "  {'reg_alpha': 10, 'reg_lambda': 0.001},\n",
       "  {'reg_alpha': 10, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 10, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 10, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 10, 'reg_lambda': 10},\n",
       "  {'reg_alpha': 10, 'reg_lambda': 100},\n",
       "  {'reg_alpha': 100, 'reg_lambda': 0.001},\n",
       "  {'reg_alpha': 100, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 100, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 100, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 100, 'reg_lambda': 10},\n",
       "  {'reg_alpha': 100, 'reg_lambda': 100}],\n",
       " 'split0_test_score': array([-0.57734023, -0.57838899, -0.5780371 , -0.57754752, -0.57853727,\n",
       "        -0.58286974, -0.5780471 , -0.57849826, -0.57773592, -0.5779808 ,\n",
       "        -0.57855425, -0.58293395, -0.57858301, -0.57752666, -0.57804036,\n",
       "        -0.57786972, -0.57882616, -0.58346064, -0.57736795, -0.57788976,\n",
       "        -0.57710913, -0.57787751, -0.57831502, -0.58358692, -0.58257678,\n",
       "        -0.58264049, -0.58267415, -0.58291089, -0.58388968, -0.58731012,\n",
       "        -0.61618192, -0.61618202, -0.61621702, -0.61595596, -0.61630901,\n",
       "        -0.61704629]),\n",
       " 'split1_test_score': array([-0.58291342, -0.58325801, -0.58308201, -0.58233753, -0.58359924,\n",
       "        -0.58792826, -0.58303375, -0.58258081, -0.58336609, -0.58275515,\n",
       "        -0.58390969, -0.58798487, -0.58239714, -0.5828199 , -0.58292626,\n",
       "        -0.58232385, -0.58312048, -0.58813687, -0.5827727 , -0.58198322,\n",
       "        -0.58265004, -0.58193962, -0.58372737, -0.58769413, -0.58649064,\n",
       "        -0.58663405, -0.58607791, -0.58694723, -0.58753908, -0.59134121,\n",
       "        -0.62337908, -0.62337918, -0.62338021, -0.62345187, -0.6232945 ,\n",
       "        -0.62457977]),\n",
       " 'split2_test_score': array([-0.58490053, -0.58497389, -0.58394523, -0.58451793, -0.58530502,\n",
       "        -0.58841788, -0.58502687, -0.58399402, -0.58345367, -0.58433834,\n",
       "        -0.5860254 , -0.5883115 , -0.58472084, -0.58469743, -0.58453448,\n",
       "        -0.58419419, -0.58563712, -0.58928298, -0.58445839, -0.58444208,\n",
       "        -0.58413257, -0.585137  , -0.58594272, -0.58916922, -0.58880028,\n",
       "        -0.58898856, -0.58830257, -0.5886862 , -0.58908359, -0.59246168,\n",
       "        -0.62176178, -0.62176186, -0.62176274, -0.62133188, -0.62135779,\n",
       "        -0.62259476]),\n",
       " 'split3_test_score': array([-0.58790589, -0.58816717, -0.58828269, -0.58697275, -0.58779601,\n",
       "        -0.59165959, -0.58861019, -0.58809892, -0.58752258, -0.58802285,\n",
       "        -0.58780657, -0.59202008, -0.58788909, -0.58742632, -0.58827315,\n",
       "        -0.58831781, -0.5874689 , -0.5924043 , -0.58747034, -0.58695724,\n",
       "        -0.58651197, -0.58701414, -0.58879737, -0.59229385, -0.59085287,\n",
       "        -0.59052052, -0.59125293, -0.59083611, -0.59181773, -0.59532746,\n",
       "        -0.62555164, -0.62536802, -0.62546844, -0.62532896, -0.62554598,\n",
       "        -0.62661316]),\n",
       " 'split4_test_score': array([-0.58856587, -0.58870708, -0.58874348, -0.58839208, -0.58689459,\n",
       "        -0.59193536, -0.58819317, -0.58797739, -0.58857187, -0.58840828,\n",
       "        -0.58743375, -0.59151527, -0.58823163, -0.58850638, -0.58796178,\n",
       "        -0.58830015, -0.58751669, -0.59188287, -0.58714241, -0.58655536,\n",
       "        -0.58516001, -0.5865391 , -0.58715688, -0.59236755, -0.59101123,\n",
       "        -0.59105557, -0.59083703, -0.59095755, -0.59136784, -0.59556463,\n",
       "        -0.62248615, -0.62248623, -0.62248706, -0.62308399, -0.62265193,\n",
       "        -0.62356876]),\n",
       " 'mean_test_score': array([-0.58432493, -0.58469878, -0.58441784, -0.58395329, -0.58442628,\n",
       "        -0.58856196, -0.584582  , -0.58422965, -0.58412975, -0.58430084,\n",
       "        -0.58474577, -0.58855295, -0.58436411, -0.58419507, -0.58434699,\n",
       "        -0.58420089, -0.58451369, -0.58903336, -0.58384216, -0.58356535,\n",
       "        -0.58311262, -0.5837013 , -0.58478773, -0.58902213, -0.58794617,\n",
       "        -0.58796765, -0.58782873, -0.58806742, -0.58873942, -0.59240083,\n",
       "        -0.62187208, -0.62183543, -0.62186306, -0.62183046, -0.62183179,\n",
       "        -0.62288051]),\n",
       " 'std_test_score': array([0.00404767, 0.00374328, 0.00390817, 0.00381528, 0.0032737 ,\n",
       "        0.0032802 , 0.0038618 , 0.00359554, 0.00382527, 0.00382249,\n",
       "        0.00338427, 0.00324811, 0.0036    , 0.00388943, 0.0037499 ,\n",
       "        0.00393344, 0.00326505, 0.0032064 , 0.0036741 , 0.00334455,\n",
       "        0.00325716, 0.00340911, 0.00363447, 0.00326259, 0.00314713,\n",
       "        0.00307414, 0.00318229, 0.00297476, 0.00287973, 0.00302047,\n",
       "        0.00311706, 0.00307425, 0.00308482, 0.00320002, 0.00307687,\n",
       "        0.00320676]),\n",
       " 'rank_test_score': array([11, 18, 14,  5, 15, 26, 17,  9,  6, 10, 19, 25, 13,  7, 12,  8, 16,\n",
       "        29,  4,  2,  1,  3, 20, 28, 22, 23, 21, 24, 27, 30, 35, 33, 34, 31,\n",
       "        32, 36], dtype=int32),\n",
       " 'split0_train_score': array([-0.50625326, -0.50650737, -0.50676367, -0.50858502, -0.51657675,\n",
       "        -0.54197521, -0.50645387, -0.50698185, -0.50653967, -0.50848951,\n",
       "        -0.51681417, -0.54223089, -0.50614285, -0.50590045, -0.50604487,\n",
       "        -0.50799941, -0.51719651, -0.54230864, -0.50500874, -0.50527391,\n",
       "        -0.50547648, -0.5070711 , -0.5167737 , -0.54346298, -0.53821798,\n",
       "        -0.53821057, -0.53838427, -0.53871024, -0.54190493, -0.55634943,\n",
       "        -0.61280516, -0.61280528, -0.61281698, -0.61269241, -0.61336517,\n",
       "        -0.6142713 ]),\n",
       " 'split1_train_score': array([-0.50538648, -0.50550851, -0.50490212, -0.50684268, -0.51480703,\n",
       "        -0.54120678, -0.50536281, -0.50540974, -0.50480658, -0.50683218,\n",
       "        -0.51518018, -0.54134379, -0.50499956, -0.50391967, -0.50526662,\n",
       "        -0.50690537, -0.51437189, -0.54130323, -0.50459617, -0.50337818,\n",
       "        -0.50367215, -0.50553856, -0.51554685, -0.54256426, -0.53758001,\n",
       "        -0.53735259, -0.53757839, -0.53779474, -0.54106099, -0.5552155 ,\n",
       "        -0.61196036, -0.61196047, -0.61196164, -0.61199335, -0.61186984,\n",
       "        -0.61324355]),\n",
       " 'split2_train_score': array([-0.50475119, -0.50512489, -0.50582133, -0.50683945, -0.51571202,\n",
       "        -0.54073966, -0.50491339, -0.5052109 , -0.5046219 , -0.50609045,\n",
       "        -0.51564112, -0.54125885, -0.50466821, -0.50423247, -0.50542007,\n",
       "        -0.50705472, -0.51551018, -0.54162582, -0.50505891, -0.5040321 ,\n",
       "        -0.50428906, -0.50610553, -0.51675175, -0.5426374 , -0.5375268 ,\n",
       "        -0.5373702 , -0.53699891, -0.53729412, -0.5404598 , -0.55525579,\n",
       "        -0.61272686, -0.61272697, -0.61272813, -0.612637  , -0.61230726,\n",
       "        -0.61356342]),\n",
       " 'split3_train_score': array([-0.50389463, -0.50447103, -0.5045404 , -0.50582133, -0.51513362,\n",
       "        -0.5416666 , -0.50360886, -0.5039211 , -0.50364997, -0.50636284,\n",
       "        -0.51501128, -0.54136807, -0.50431264, -0.50363594, -0.50454699,\n",
       "        -0.50556641, -0.51534505, -0.54174976, -0.50383005, -0.50432643,\n",
       "        -0.50423074, -0.50602741, -0.51578116, -0.54256154, -0.53742067,\n",
       "        -0.53765276, -0.53754845, -0.53789366, -0.54056468, -0.55493623,\n",
       "        -0.61122154, -0.61105136, -0.61116518, -0.61100452, -0.61138818,\n",
       "        -0.6124464 ]),\n",
       " 'split4_train_score': array([-0.5031734 , -0.50410456, -0.50477613, -0.50662252, -0.51373404,\n",
       "        -0.53967936, -0.50329149, -0.50315669, -0.50321545, -0.50548223,\n",
       "        -0.51440082, -0.53931036, -0.50372535, -0.50343226, -0.50399613,\n",
       "        -0.50410615, -0.51421173, -0.53950596, -0.5026412 , -0.5020789 ,\n",
       "        -0.50263646, -0.50444718, -0.51523473, -0.54105439, -0.53581988,\n",
       "        -0.53564157, -0.53573493, -0.53592442, -0.53891985, -0.55374876,\n",
       "        -0.61110297, -0.61110309, -0.61110431, -0.61163234, -0.61113182,\n",
       "        -0.61247034]),\n",
       " 'mean_train_score': array([-0.50469179, -0.50514327, -0.50536073, -0.5069422 , -0.51519269,\n",
       "        -0.54105352, -0.50472608, -0.50493606, -0.50456671, -0.50665144,\n",
       "        -0.51540952, -0.54110239, -0.50476972, -0.50422416, -0.50505493,\n",
       "        -0.50632641, -0.51532707, -0.54129868, -0.50422701, -0.5038179 ,\n",
       "        -0.50406098, -0.50583796, -0.51601764, -0.54245611, -0.53731307,\n",
       "        -0.53724554, -0.53724899, -0.53752344, -0.54058205, -0.55510114,\n",
       "        -0.61196338, -0.61192944, -0.61195525, -0.61199192, -0.61201245,\n",
       "        -0.613199  ]),\n",
       " 'std_train_score': array([0.00108308, 0.00083968, 0.00082528, 0.00090309, 0.00094501,\n",
       "        0.00080445, 0.00116035, 0.00131769, 0.00115061, 0.00101747,\n",
       "        0.00080689, 0.00096314, 0.00080601, 0.00088044, 0.00071235,\n",
       "        0.00135438, 0.0010663 , 0.00095335, 0.00090684, 0.00106226,\n",
       "        0.00092357, 0.00085464, 0.00063263, 0.00077901, 0.00079712,\n",
       "        0.00085987, 0.00087658, 0.00091963, 0.00097545, 0.00083083,\n",
       "        0.00071863, 0.00075604, 0.00073325, 0.0006342 , 0.00078769,\n",
       "        0.00069021])}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.583113 using {'reg_alpha': 1, 'reg_lambda': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcXGWd7/HPr/at9yQQ0gkQkgCJMCANBHEhOEgQBe/VqzCORgcBFy6IV1ncJjeignJRuYIYdmZkUUYhIIs4wshliCQBBiFAEgOazt77vtSp3/3jnO5UV1dVL9XV6+/9etWr+5x6zunnpOB863mec84jqooxxhgzWr6JroAxxpipzYLEGGNMQSxIjDHGFMSCxBhjTEEsSIwxxhTEgsQYY0xBLEiMMcYUxILEGGNMQSxIjDHGFCQw0RUYD7NmzdLDDjtsoqthjDFTyqZNm+pUdfZQ5WZEkBx22GFs3LhxoqthjDFTioj8dTjlrGvLGGNMQSxIjDHGFMSCxBhjTEEsSIwxxhTEgsQYY0xBLEiMMcYUxILEGGNMQWbEfSTGGDOZqSopxyGVckglHVJOkpTj4CTdn33LKcchlUzi9K3zyjqOgzoOTl85bzvHSXL0u08jEk8Utf4WJMaYKUdTqSwnU+/3rCfjtBNsKnWg7HBP2P3buvvOdsLu3y7jBJ8asO9UjvVO0f6tFiz7OwsSY0xh+r/tDjpppp/4Mr7lpp2MnbQyfSdNdYZ5MnaSOMmBJ2PHcdBURvnMk3Mq97fvlJNCNTVu/34+vx+fP+D99F6BAP709d6y+P34/QECoRC+QAyft9xXxufz4wu42/n71vkPLLvb96133/MHMv52tm0DAcTnSysbwBdw/3YkUVL0fyMLEmMyDPxG2/cNMtdJc/BJctC33wEnzIwyebor+t7L+u23/2R/YN+5vv0W89tuJhGfe6Ic1gkz0F8mEAoPPAmmnzQDA0/YPp8/4wR74KSZebLv//tpf8vn92XUK/cJ2+f3IyLj9u83VVmQmIK433bTTnq5vmVma/LnOmlmnLA182Sc+S03T1dEvr7jzO6KvnqhOm7/fgNOnIEsJz2fb9D6YDiCP5Z2Mk47UfsHnDDzfYP1I758J+O0E26WMuL3eWUDAwJDfHb9zkxkQZLH1g3P8+Zzf2Tpe08nHIsTCIXG7W/3n6CznYzzfIPNdiLN2V2R3nec40Tef9LO+PbbVy9NjV8XQ/+33UEnxtwnzEAwgIQHf9vN+u13iO6KrCdsn3/QydSXZd9ZT8Y+n33bNdOCBUkebzz3R7Y8/yxvPv/sRFdlVLI3+TNOehlN/r5vu5ldEZn9uzlPxlm6IgadjDPrlbbvfCds+7ZrzORkQZLH33/ui5x0zkdJOQ7dHe0ke3pgFN8gR/uls/+Enf7tN6N7YlDfsX3bNcaMMwuSPKKJEqLjcMWDMcZMZdZXYIwxpiAWJMYYYwpiQWKMMaYgRQ0SEVkpIm+KyDYRuSpHmY+LyGYReU1E7vXWHSciz3vrXhGRT6SVP1xE/uTt8wERGb9rco0xxgxStCARET9wE3AWsBQ4X0SWZpRZDFwNnKqqy4Ave291AJ/21q0Efiwi5d571wE/UtVFQCNwQbGOwRhjzNCK2SI5CdimqttVtQe4Hzg3o8yFwE2q2gigqvu8n1tUdav3+y5gHzBb3OtZTwce9La/G/hIEY/BGGPMEIoZJPOAHWnLtd66dEuAJSLynIisF5GVmTsRkZOAEPAXoApoUtVknn0aY4wZRxN9H0kAWAycBlQDfxSRY1S1CUBE5gL/AqxS1dRIbrATkYuAiwAWLFgwxtU2xhjTp5gtkp3A/LTlam9dulpgnar2qupbwBbcYEFESoHfAt9Q1fVe+XqgXEQCefYJgKquVdUaVa2ZPXv2mByQMcaYwYoZJBuAxd5VViHgPGBdRpmHcFsjiMgs3K6u7V753wD3qGrfeAiqqsDTwMe8VauAh4t4DMYYY4ZQtCDxxjEuAZ4EXgd+qaqvicgaETnHK/YkUC8im3ED4muqWg98HHgv8BkRedl7HedtcyXwFRHZhjtmcnuxjsEYY8zQRMdx7oWJUlNToxs3bpzoahhjzJQiIptUtWaocnZnuzHGmIJYkBhjjCmIBYkxxpiCWJAYY4wpiAWJMcaYgliQGGOMKYgFiTHGmIJYkBhjjCmIBYkxxpiCWJAYY4wpiAWJMcaYgliQGGOMKYgFiTHGmIJYkBhjjCmIBYkxxpiCWJAYY4wpSFGDRERWisibIrJNRK7KUebjIrJZRF4TkXvT1j8hIk0i8mhG+btE5K0sMycaY4yZAIFi7VhE/MBNwBlALbBBRNap6ua0MouBq4FTVbVRROak7eKHQAy4OMvuv5Y+l7sxxpiJU8wWyUnANlXdrqo9wP3AuRllLgRuUtVGAFXd1/eGqv470FrE+hljjBkDxQySecCOtOVab126JcASEXlORNaLyMph7vu7IvKKiPxIRMJjUVljjDGjM9GD7QFgMXAacD5wq4iUD7HN1cBRwIlAJXBltkIicpGIbBSRjfv37x+7GhtjjBmgmEGyE5iftlztrUtXC6xT1V5VfQvYghssOanqbnV1A3fidqFlK7dWVWtUtWb27NmjPghjjDH5FTNINgCLReRwEQkB5wHrMso8hNsaQURm4XZ1bc+3UxGZ6/0U4CPAq2NbbWOMMSNRtKu2VDUpIpcATwJ+4A5VfU1E1gAbVXWd994HRGQz4OBejVUPICLP4nZhJUSkFrhAVZ8EfiEiswEBXgY+X6xjMMYYMzRR1YmuQ9HV1NToxo0bJ7oaxhgzpYjIJlWtGarcRA+2G2OMmeIsSIwxxhTEgsQYY0xBLEiMMcYUxILEGGNMQYp2+a8xZvrq7e2ltraWrq6uia6KGQORSITq6mqCweCotrcgMcaMWG1tLSUlJRx22GG49wabqUpVqa+vp7a2lsMPP3xU+7CuLWPMiHV1dVFVVWUhMg2ICFVVVQW1Li1IjDGjYiEyfRT6WVqQGGOMKYgFiTHG5PHMM8/woQ99qOAy+agql156KYsWLeLYY4/lxRdfzFpu06ZNHHPMMSxatIhLL72UvkdcNTQ0cMYZZ7B48WLOOOMMGhsbAXjjjTc45ZRTCIfDXH/99aOu31AsSIwxU5qqkkqlJroaBXn88cfZunUrW7duZe3atXzhC1/IWu4LX/gCt956a3/ZJ554AoBrr72W97///WzdupX3v//9XHvttQBUVlZy44038tWvfrWo9bertowxBfnfj7zG5l0tY7rPpYeU8s8fXpbz/bfffpszzzyTk08+mU2bNnHFFVdwyy230N3dzRFHHMGdd95JIpHgscce4ytf+QrxeJxTTz2V7du38+ijj2bd5wsvvMBll11GV1cX0WiUO++8kyOPPHJAmdWrV/OXv/yFbdu2UVdXxxVXXMGFF14IQFtbGx/72Md49dVXOeGEE/jXf/1XRIQ1a9bwyCOP0NnZybve9S5+/vOfDxqTePjhh/n0pz+NiLB8+XKamprYvXs3c+fO7S+ze/duWlpaWL58OQCf/vSneeihhzjrrLN4+OGHeeaZZwBYtWoVp512Gtdddx1z5sxhzpw5/Pa3vx3xZzAS1iIxxkxJW7du5Ytf/CL/8R//we23387vf/97XnzxRWpqarjhhhvo6uri4osv5vHHH2fTpk0MNVPqUUcdxbPPPstLL73EmjVr+PrXv5613CuvvMIf/vAHnn/+edasWcOuXbsAeOmll/jxj3/M5s2b2b59O8899xwAl1xyCRs2bODVV1+ls7OzP8huueUWbrnlFgB27tzJ/PkH5gGsrq5m586B8wDu3LmT6urqrGX27t3bHzoHH3wwe/fuHfa/41iwFokxpiD5Wg7FdOihh7J8+XIeffRRNm/ezKmnngpAT08Pp5xyCm+88QYLFy7svzfi/PPPZ+3atTn319zczKpVq9i6dSsiQm9vb9Zy5557LtFolGg0yooVK3jhhRcoLy/npJNO6j/RH3fccbz99tu8+93v5umnn+YHP/gBHR0dNDQ0sGzZMj784Q/z+c8XZyolERn3K+osSIwxU1I8HgfcMZIzzjiD++67b8D7L7/88oj2961vfYsVK1bwm9/8hrfffpvTTjsta7nMk3Tfcjgc7l/n9/tJJpN0dXXxxS9+kY0bNzJ//nxWr16d9X6NefPmsWPHjv7l2tpa5s2bN6hMbW1t1jIHHXRQf1fY7t27mTNnzoiOvVBF7doSkZUi8qaIbBORq3KU+biIbBaR10Tk3rT1T4hIk4g8mlH+cBH5k7fPB7xpfI0xM9Ty5ct57rnn2LZtGwDt7e1s2bKFI488ku3bt/P2228D8MADD+TdT3Nzc/+J+a677spZ7uGHH6arq4v6+nqeeeYZTjzxxJxl+0Jj1qxZtLW18eCDD2Ytd84553DPPfegqqxfv56ysrIB4yMAc+fOpbS0lPXr16Oq3HPPPZx77rn92999990A3H333f3rx0vRgkRE/MBNwFnAUuB8EVmaUWYxcDVwqqouA76c9vYPgU9l2fV1wI9UdRHQCFxQhOobY6aI2bNnc9ddd3H++edz7LHH9ndrRaNRbr75ZlauXMkJJ5xASUkJZWVlOfdzxRVXcPXVV3P88ceTTCZzljv22GNZsWIFy5cv51vf+haHHHJIzrLl5eVceOGFvOMd7+DMM88cEDrpYyQf/OAHWbhwIYsWLeLCCy/k5ptv7i933HHH9f9+880387nPfY5FixZxxBFHcNZZZwFw1VVX8dRTT7F48WJ+//vfc9VV7vf2PXv2UF1dzQ033MA111xDdXU1LS1je2EEFHGqXRE5BVitqmd6y1cDqOr308r8ANiiqrfl2MdpwFdV9UPesgD7gYO9OeEH/I1cbKpdY8bW66+/ztFHHz3R1RhSW1sbiUQCVeVLX/oSixcv5vLLLx/1/lavXk0ikSj65bQTIdtnOhmm2p0H7EhbrvXWpVsCLBGR50RkvYisHGKfVUCTqvZ9Xci2TwBE5CIR2SgiG4e6WsMYMz3deuutHHfccSxbtozm5mYuvvjiia7StDTRg+0BYDFwGlAN/FFEjlHVpkJ3rKprgbXgtkgK3Z8xZuq5/PLLB7VA7rzzTn7yk58MWHfqqady0003Dbm/1atXj2X1po1iBslOYH7acrW3Ll0t8CdV7QXeEpEtuMGyIcc+64FyEQl4rZJs+zTGmJw++9nP8tnPfnaiqzGtFLNrawOw2LvKKgScB6zLKPMQbmsEEZmF29W1PdcO1R3QeRr4mLdqFfDw2FbbGGPMSBQtSLwWwyXAk8DrwC9V9TURWSMi53jFngTqRWQzbkB8TVXrAUTkWeBXwPtFpFZE+gbUrwS+IiLbcMdMbi/WMRhjjBlaUcdIVPUx4LGMdd9O+12Br3ivzG3fk2Of24GTxramxhhjRsuetWWMMaYgFiTGGJPHVJiP5Fe/+hXLli3D5/MxEffMWZAYY6Y0m48E3vGOd/DrX/+a9773veNZ7X4TfR+JMWaqe/wq2PPnsd3nwcfAWdfmfNvmIxk4H8lEP2XAWiTGmCnJ5iMZXGaiWIvEGFOYPC2HYrL5SCYPCxJjzJRk85EMLjNRRtS1JSI+ESktVmWMMWakbD6SiTdkkIjIvSJSKiJx4FVgs4h8rfhVM8aYodl8JPCb3/yG6upqnn/+ec4++2zOPDPvzBpjbsj5SETkZVU9TkQ+CbwTuArYpKrHjkcFx4LNR2LM2LL5SGw+knTD6doKikgQ+AiwzntSrz2W3Rgz6dl8JONjOIPtPwfeBv4Ld76QQ4Gxn6vRGGPGmM1HMj6GDBJVvRG4MW3VX0VkRfGqZIwxxWPzkYy94Qy2X+YNtouI3C4iLwKnj0PdjDHGTAHDGSP5J1VtAT4AVACfAibmDiRjjDGTznCCpO/umw8C/6Kqr6Wty7+hyEoReVNEtonIVTnKfFxENovIayJyb9r6VSKy1XutSlv/jLfPl73XnOHUxRhjTHEMZ7B9k4j8DjgcuFpESoAhH7UpIn7gJuAM3LnZN4jIOlXdnFZmMXA1cKqqNvaFgohUAv8M1OBeIbbJ27bR2/STqmrX8xpjzCQwnBbJBbj3jpyoqh1ACBjOSNVJwDZV3a6qPcD9QOZtmBcCN/UFhKru89afCTylqg3ee08BK4fxN40xZkxNpvlIvvGNbzB//nwSicSo/1YxDBkkqpoCqoFvisj1wLtU9ZVh7HsesCNtudZbl24JsEREnhOR9SKycpjb3ul1a31LMh98Y4yZUWbSfCQf/vCHeeGFF8a5dkMbsmtLRK4FTgR+4a26VEROUdXsz1ge+d9fDJyGG1Z/FJFjhtjmk6q60+ti+zfcwf97stT7IuAigAULFoxBVY0x2Vz3wnW80fDGmO7zqMqjuPKkK3O+PxPnIwH65yKZbIbTtfVB4AxVvUNV78DtYhpOG24nMD9tudpbl64W7255VX0L2IIbLDm3VdW+n63AvbhdaIOo6lpVrVHVmtmzZw+jusaYqWSmzUcymQ33MfLlQIP3e+6nng20AVgsIofjhsB5wD9klHkIOB+3q2oWblfXduAvwPdEpMIr9wHcgf4AUK6qdd5jWz4E/H6Y9THGFEG+lkMx2Xwkk8dwguT7wEsi8jTuZb/vxR18z0tVkyJyCfAk4AfuUNXXRGQNsFFV13nvfUBENgMO8DVVrQcQke/ghhHAGlVt8J5A/KQXIn7cELl1BMdrjJkmZtp8JJPZcAbb7wOWA7/GHZM4BXh2ODtX1cdUdYmqHqGq3/XWfdsLEdT1FVVdqqrHqOr9adveoaqLvNed3rp2VT1BVY9V1WWqepmqOiM+amPMtDFT5iOZzIY1sZWq7lbVdd5rD7C+yPUyxphhmUnzkVxxxRVUV1fT0dFBdXX1pHmI5JDzkWTdSGSHqs4fuuTkYPORGDO2bD4Sm48k3Yim2k1j85EYYyY9m49kfORskYjI/yV7YAiwSlWnzNzt1iIxZmxNlRZJNoXMRzKdFdIiyXfVVr4zr52VjTFTks1HMvZyBomq3j2eFTHGGDM1jXaMxBhjjAEsSIwxxhTIgsQYY0xBhjNn+41ZXt8Rkcy5RYwxZtoZj/lI3njjDU455RTC4TDXX3/9qPczUYbzrK0IcBTwK2/5o8BbwN+JyApV/XKxKmeMMUNRVVQVn2/qdrBUVlZy44038tBDD010VUZlOEFyLO5UuA6AiPwM91lb7wb+XMS6GWOmgD3f+x7dr4/tfCTho4/i4ByPcYfpNx/JnDlzmDNnDr/97W8L/JebGMOJ8AogfV7HOFDpBUt3UWpljDFDmE7zkUx1w2mR/AB4WUSe4cBj5L/nPdLd5gIxZobL13IoJpuPZPIYMkhU9XYReYwDMxF+XVV3eb9/rWg1M8aYPKbTfCRT3XBHp04E3uO9TihedYwxZmSmw3wkU91wLv+9FrgM2Oy9LhWR7w1n5yKyUkTeFJFtIpJ1VkUR+biIbBaR10Tk3rT1q0Rkq/dalbb+BBH5s7fPGyXz64ExZkaZDvOR7Nmzh+rqam644QauueYaqquraWlpGcW/xsQYcj4SEXkFOE5VU96yH3hJVY8dYjs/sAU4A6jFnTb3fFXdnFZmMfBL4HRVbRSROaq6T0QqcR8MWYP7BOJNwAlemReAS4E/AY8BN6rq4/nqYk//NWZsTZWn/9p8JMM3HvORlKf9njvSBzoJ2Kaq21W1B7gfyLyJ8ULgJlVtBFDVfd76M4GnVLXBe+8pYKWIzAVKVXW9ugl4D/CRYdbHGDPD2Hwk42M4V219H3hJRJ7mwFVbWbupMswDdqQt1wInZ5RZAiAizwF+YLWqPpFj23neqzbLemOMGeTyyy8f1AIpZD6SyTK17WQznKu27vMu/e3r3LuSsXtGVwBYDJwGVAN/FJFjxmLHInIRcBHAggULxmKXxphpwOYjGXvDCgRV3a2q67zXHmD9MDbbCaTP617trUtXC6xT1V5VfQt3TGVxnm13er/n22dfndeqao2q1syePXsY1TXGGDMao21ZDOdKqQ3AYhE5XERCwHnAuowyD+G2RhCRWbhdXduBJ4EPiEiFiFQAHwCeVNXdQIuILPeu1vo08PAoj8EYY8wYGM4YSTb5L/UCVDUpIpfghoIfuENVXxORNcBGVV3HgcDYDDjA11S1HkBEvoMbRgBrVLXB+/2LwF1AFHjcexljjJkgOYNERP4v2QNDGHgVV06q+hjuJbrp676d9rsCX/FemdveAdyRZf1G4B3D+fvGGGOKL1/X1kbc+zcyXxuB/1n8qhljzMSb6PlInnjiCY488kgWLVrEtddeO+q/UUw5WySqenfmOhE52BtsN8aYSWE6z0fiOA5f+tKXeOqpp6iurubEE0/knHPOYenSpRNU0+xGOkbyGPDOYlTEGDM1PfvLLdTtaBvTfc6an+A9H1+S8/2ZMh/JCy+8wKJFi1i4cCEA5513Hg8//PCkC5KRRrg918oYMynMhPlIdu7cyfz5B+6EqK6uZufOrHc8TKiRtkhuLUotjDFTVr6WQzHZfCSTx4iCRFVvLlZFjDFmJGbCfCTz5s1jx44DT4uqra3tf9T9ZDJ1R6eMMYbpPR/JiSeeyNatW3nrrbfo6enh/vvv55xzzhnRPsbDaG9INMaYSSF9PpLu7m4ArrnmGpYsWdI/H0k8Hs97wgd3PpJVq1ZxzTXXcPbZZ+cs1zcfSV1dXf98JFu2bMlaNn0+koMPPnjQfCQAn//859mzZw81NTW0tLTg8/n6x1pKS0v56U9/yplnnonjOPzTP/0Ty5YtG+k/UdENOR/JdGDzkRgztmw+EpuPJJ11bRljpi2bj2R8WIvEGDNiU6VFkk0h85FMZ4W0SGyMxBgzo9h8JGPPuraMMcYUxILEGGNMQSxIjDHGFMSCxBhjTEGKGiQislJE3hSRbSJyVZb3PyMi+0XkZe/1ubT3rhORV73XJ9LW3yUib6Vtc1wxj8EYM7NN1vlI3nrrLU4++WQWLVrEJz7xCXp6ekb99wtVtCARET9wE3AWsBQ4X0SyPfv4AVU9znvd5m17Nu7j6o8DTga+KiKladt8LW2bkT1QxxgzragqqVRqoqtRkL75SDJvdOybj+Txxx9n8+bN3HfffWzevBmAK6+8kssvv5xt27ZRUVHB7bffPhFVB4p7+e9JwDZV3Q4gIvcD5wKbh7HtUuCPqpoEkiLyCrAS+GWxKmuMGZ2n71rLvr9uH9N9zjl0ISs+c1HO92f6fCRHH300f/jDH7j33nsBWLVqFatXr+YLX/jCyP6hx0gxu7bmATvSlmu9dZk+KiKviMiDItL34P3/AlaKSExEZgErgPlp23zX2+ZHIhIetEdjzLQ3k+cjqa+vp7y8nEAgMGD9RJnoGxIfAe5T1W4RuRi4GzhdVX8nIicC/wnsB54HHG+bq4E9QAhYC1wJrMncsYhcBFwEsGDBgmIfhzEzVr6WQzHZfCSTRzFbJDsZ2Iqo9tb1U9V6Ve32Fm8DTkh777veGMgZuDMzbvHW71ZXN3AnbhfaIKq6VlVrVLVm9uzZY3ZQxpjJIXM+kpdffpmXX36ZzZs3j2q8oG8+kldffZVHHnkk57who5mP5MEHH+TPf/4zF1544ZjMR1JVVUVTUxPJZHLA+olSzCDZACwWkcNFJAScB6xLLyAic9MWzwFe99b7RaTK+/1Y4Fjgd+nbiPvpfQR4tYjHYIyZ5GbifCQiwooVK/r3d/fdd3PuueeOaN9jqWhdW6qaFJFLgCcBP3CHqr4mImuAjaq6DrhURM4BkkAD8Blv8yDwrJf0LcA/egPvAL8Qkdm4rZSXgenTPjTGjNhMnY/kuuuu47zzzuOb3/wmxx9/PBdccMGI/t3Gkj391xgzYlPl6b82H8nw2XwkxhiThc1HMj6sRWKMGbGp0iLJxuYjyc7mIzHGjDtVHXQF01Rg85EMVmiDwrq2jDEjFolEqK+vL/gEZCaeqlJfX08kEhn1PqxFYowZserqampra4e8W9xMDZFIpP9mytGwIDHGjFgwGOy/Y9wY69oyxhhTEGuRGGPMJKeq9HQ5dLf30t2RpLuj72eSrrTf+9e3H1j3P66uoXRWtKj1syAxxphxoCmlpys5MADaB4bCgd97vTLu7z0dSfJd1yA+IRwLeK8gkXiQstlRwrEg/kDxO54sSIwxZphGEgZdGa2Hns78YeDzCeG4GwThWIBIIkjZnNiAgAjHAkS8n+llg2H/hF6KbUFijJlRNKV0d2a2AAa2Bro6khkB4f3sTEK+MPDLgJN+tCRE+UExIrEA4XhwUCgc+DnxYVAICxJjzJSTSik9neljAuljBdnCYQRhEBC3e8g7wcdKQ1QcHBtw0u9vHcQHhkIg5JuyYVAICxJjzIRIpZSejjwB0D54rKDv/Z7OZN59jzoM4kECwZkZBoWwIDHGjFrKSbndRO05uoeyDCD3BURPl5N33/6Ab8BJP14epvKQ+OCxgsyuonjAwmCcWZAYM8OlnFSWS0l7s4RDekC463qHCoOgr/8EH4kFSJSHqTokMTAA4gPHCvoCIhDyj9O/gCmUBYkx04DjpPJeStrfOmgf/H5vd/4wCPSFgTdYXFIZYVZ1YmD3UJZWQTgWIBC0MJgJihokIrIS+AnuDIm3qeq1Ge9/BvghB+Zy/6mq3ua9dx3QN03Zd1T1AW/94cD9QBWwCfiUqvYU8ziMGQ9OMjXgBN+V5aSf9TLTjiTJocIg5Bvwrb+kKsLs+YkBJ/1sVxJFYkH8QXsAhsmvaEEiIn7gJuAMoBbYICLrVHVzRtEHVPWSjG3PBt4JHAeEgWdE5HFVbQGuA36kqveLyC3ABcDPinUcxozEkGHQPrh7qO+9ZE8q774DYX//4HE4FqR0VjR3AMTT1kUDFgamqIrZIjkJ2Kaq2wFE5H7gXCAzSLJZCvzRm6c9KSKvACtF5FfA6cA/eOXuBlZjQWLGkNObyv7YiYzB4rEIg7LZ0f4uo0HdQxkDyeNxh7Ixo1HMIJkH7EhbrgVOzlLuoyLyXmALcLmq7gD+C/hnEfk/QAxYgRtAVUCTFzB9+5xXpPqbKSzZ6ww46ee+kmjw84mSvfnDIBj2DzjBWxii9XzQAAAYlElEQVSYmW6iB9sfAe5T1W4RuRi3hXG6qv5ORE4E/hPYDzwP5O8EziAiFwEXASxYsGBsa23GRbLHybiSaPD9BdmeS9TdkcQZKgwiB8IgEgtQnuVRFAMeV+H9DMUC+P0WBsakK2aQ7ATmpy1Xc2BQHQBVrU9bvA34Qdp73wW+CyAi9+K2WOqBchEJeK2SQftM234tsBbcOdsLPRgzcqpKsjfzaqJsTy3tzdJ6SOIk84dBKOIfMFjs3nCW50qivnCIBvBZGBgzZooZJBuAxd5VVjuB8zgwtgGAiMxV1d3e4jnA6956P1CuqvUicixwLPA7VVUReRr4GO6VW6uAh4t4DDOeqpLsSQ3sHspxf0G21kEqmT/DQ9HAgJZA5dx4zvsL0lsHoajfwsCYSaJoQaKqSRG5BHgS9/LfO1T1NRFZA2xU1XXApSJyDpAEGoDPeJsHgWe9O1NbgH9MGxe5ErhfRK4BXgJuL9YxTBeqSm+3k/9S0vbcdyKnnHzPr4ZwdOCJPl4ezhkA6ZeahqIBfD67+9iYsdaT7KGhcSeN+3ZwxBE1hCKxov490XzPNZ4mampqdOPGjRNdjYIMDIMDdx0P90F1Iw2DvhbBUIPHFgbGFJeq0tFST+Pev9G8bwet+3bR2bCP7vr99DY2oE0tSHMrgdYOQm09RNt7SXQoQW9UOfSrtRxxzHtG9bdFZJOq1gxVbqIH22e8VEppb+qmtaGL1vqu/p9tjd3ePQjeXAYdSVKpkYVBoiKSNwz67jUIRQKIhYExRaeqOO3ttO3bSdO+WlrqaunYv4euhv30NNSTamyCllZ8Le2EWrsIt/US73D6Q8EHlHkvgBTQHvPRFQ/QUxKm96AKestKaC0vI1hRSbhqFu+Yu7Dox2VBUmROMkVb44GQaKnvoq0vMBq6aGvoHhQQkUSQksoIkXiA0qpI9tZAfOBD6ywMjBlfqkqqvYPexnpa9++ied8O2vfvprN+H90NdTiNjWhTM9LSTqC1g3BbD9H2ZH8ogNuH3xcMKYG2KHTEA3THQ7TNitNyRIK6shICFZUEK6uIVs0hWl5FIBrH7w8jqgQ6moi01RNob0Q7G5GuJgLdTYR6dxLd00JH5yf7g6dYLEgK1NvjDGhJDPi9oYv25u6Bcx8IxMvClFRGOOjwMhadEKGkyntVuq9g2J5PZMx46gsFp6mR3vo6mvfvpG3/Ljrq9tDVUEdvYz2pxmZoaSXQ0kGotYtIe5JARpdxyHulBNoi0Bbz0ZkI0lsRIXlYGZSW4C8vJ1hZSbisjHA8jj8SIRAIEgainW2EOhrRjr5AaCbUu5tw8k3i+1sp2ddKhN6cx+Go0CIltPsSdPpL6Y5UIqn8j9wfCxYkw1RX28rOLU1uKyKtddHVNvBD9fmERKUbFPOPrnDDoS8kqiIkKiJ2U5oxReSGQjtOUxNOYyPd9ftp3b+L9rrddNbto6exHqexCW1uxtfSTrC1i0hbD/4s44hhICjQGnVfHfEAPWVhktVlpMoS+MpKCJQkCJXGCUQjBMMhggE/UUkR6+qkpLMFX1cTvu4mQj0tRJK7iKdaiTe042vI3VXdrmFaSdDmK6EjUEJrqJrdoTJS4QqIluOLVxBMVBFOVBEtqyJRPpvSitlEE2VU+PxUFPHfNxsLkjxe/v3f+PMztSQqIuza2gS4j8UuqYxQWhVh1oKS/lZESZW7LlYWtsFnY8ZIfyg0NvYHQ2fdPtrqdtNZv5fu+v0kGxtJNTUjzW3egHN31lAACAl0R6E9Cq1RoSsRpHduhFRJOZTE8JfECCYi+CJh9wkGASEqSUq7Ojmsq51wdzPBnmYiyV3EnFbCdEMH7iuDo0IzcZo1QZsvQaO/lJ7gwfQGy3Ai5RCtwBcrJxivIlRSRay0injFbErKZ1FeUsLBU+gLpwVJHi8++Vc6W3tJpZR3/fdFLDnpIGJlIZswx5hRGBAKXjAkGxrprN9Le/0euur309tQj9PUhDS34mtuJ9jWlTMU1AuFvtZCW8xH95wQTqIUElGkJIw/FsIfCRAM+YgElLj0UNbTyfyuNiq7W4knG4g6bfjwbn5NAk0D/06HhmkmTpPGaZAEnb5ZdAePoCdSRipShkYq8MUqCMYrCZW4LYRY+WzKyiqpjEc4NDL9r2y0IMlj1fdPZd/bLRx0eKnd/GZMGlUl1dbW30pID4aO+j101u+np6GOZGMD2tyCr7mdQGsnPif70wocb6C5NQotMWiLCu2VPnrjETQeQhIhJBbAH/YTCkE44FAi3VT2dHJITxsH9bRQlexk0Om6032lvNZBk8ZpJs5eTfAX36F0B0vojZaTDJej0XL80QoC8SpCpZVESquIl7mtg4p4iENjQaJBv32RzMKCJA9/wMfcReUTXQ1jiqo/FNK6j5L9wdBAZ/0+uhr2k2xoINXUBC1t+Fs68oZCa9QNhpaY24XUdih0xv048TBEA0jUjz8iBINKJJCk1NdJVW8nc3raOMrppdJJEVMdGAxJ6OwNeYGQ8LqNytlMNZsCpfSEykiFy9BIORKrJBCvJFRSSaR0FonSCiriESriIQ6JBVkaDRGaQl1Hk50FiTHTyIBQ6AuDxkacxiacpiZ6G+rpqj8w4ExzC76WdmSIUGiN9XUhCa2Hur/3xAJoLABhwR+BYNAhFuih1NfFrGQXlakU8xyHSidFeSpFxLv5OaVCCzGaOhM0d8Zp1jhNVLFHE2zzJegJluJEyklF3EFlf7ySUKKSaEkVJSWlVMaDlMdCzImFODIWpDQSnPZdR5OdBYkxk9RQoeA0NtLTUE9PYx29DQ3ePQttowqFtqiPZNQHUfCFlFAoSTTQTYnfoULdMJjnOFQ4KSpTDuVOipQGaSThBkFPgubuci8UEryhCboCJSRDZWisAolV4I9XEEpUESk50DqoiAWpjIVYGAtSEQsRC1nX0VRkQWLMOFBVUq2tg7qOnMamASHR21BPb2MDTlMjNLfmDgWf13XUN9gcOxAKLTEfHRHQCEg4RSiUJBxyiPtT/aFQ4TgsSKUod1KEnDApJ0aLuqHQ3OsFg9eF9Dfi9IbcQWVKKwjEKggkqihJJCiPhaj0AqE8FuLQ2IHfreto5rAgMWaE+kMhRythQDA01pNsbEKbWxAn+5Q6jg/ao0JLFFqiSmtMaFnQN8bgozUKXREgnCIQThEOOcQCKSpSXiikHA5xhGAqgi8ZIZWM0eskaCFBU0+cpm43FNyuoxJS4TIkVoGvtJJwvJyyeJTKuNsiKPeC4JC0gLCuIzMUCxIzow0IhcZGkhmtBPenGxTJxkaSjQ2kmlsgTyh0RIXWKDR5XUgtC/q6k3z9l6r2RBR/WAmGHWL+A6EQT/mJpEJUOhGqkhGc3ji9yRI6NUFzVwlNXQla1L3q6I1gKcTcFkK0LEFFPExlPER5rC8UgizxAqHcuo5MEVmQmGlDU6ks3UfNA1oJTlNat1JDI05zE+ToPkp5odDXhdSYIxRaYoKGUgQiKaL+FOWqlKT8xFJBQk4IXzJMoDdKaTJBNFlCuVNGa3sZze0JGjXOXyWOhisIxMopjUeoiIX6xw/cFoJ7pZG7zl1fFgsSDtijdMzkYEFiJqX+UMjVfdR04L4Fp7HBXdfcAqncodAZFTq87qPGKDQukKyh0B5RJJIiHIBSFeKpAFEnSNAJQTJKsidKykngJEvxJysItVbQomXs1Tid/hLCUbd1UB4Leq0B9+Q/22slpK+riIUojQbxW9eRmcIsSEzRpYdCtkFmp6mRZEMDTkO9GwpNTTgt7XlDoTsKHRFoiypNUWg4VGiMCa1RHy399zC4XUwdESUYFErUTzwVIOyECDhhtDdCsjeOk0zQ45TRmaykvb2S5uYqWikhEvJO+N74QUXaQHJ6d1FFWkBY15GZiYoaJCKyEvgJ7gyJt6nqtRnvfwb4IQfmXf+pqt7mvfcD4GzcR/A/BVzmTbX7DDAX955VgA+o6r5iHoc5IG8oNDbiNNSRrN/vhkJTM05zC05rB+SYQC3lc8cLOiNKm3fzWuOhQl0MmmJuKLR66/suXe0NQokGDnQdORFSPRGcZIKkU0J3spyOZCVtnVV0tFYgGqUs6j4uIxwLUpI2qJzeXdQXGpWxkHUdGTMCRQsSb971m4AzgFpgg4isU9XNGUUfUNVLMrZ9F3Aq7lztAP8PeB/wjLf8SVWd2lMeTgKaSpFqaRncddRQj1O3F6d+H8mGeq/bqBWnpQ2nrStPKCi9EaU74j0ULwZNFUJDTNgXF5qjMigUOkMQwk80FSTohPElIzi9MZxkgl6nlK5kBR1OBcnuUrQjTqA+QVmkhGgsTFlfiyA+sGWQuc66jowprmK2SE4CtqnqdgARuR84F8gMkmwUiOA+2l9w53/ZW6R6Tgt9oZBMv9qobh9O3R6cur0k6+u8cYVmNxBaO3HaM+ZKSd+fT0l6odARVberqNLtPtof97E/Bs0xoTUq3rOR3FBAhGAqQDAVQZJRNxScEnqSZXQ7ZagTR5MJtClOtKWUslAFlfG4GwKJEJX9A8xua6E8FqKyLyDiIeLWdWTMpFPMIJkH7EhbrgVOzlLuoyLyXmALcLmq7lDV50XkaWA3bpD8VFVfT9vmThFxgH8DrtFpOPG809JCsr7ebR3s24WzfxfO/r04jXXuAHNTkxsILe047d04Hb15Q8GJKD0RpSsC7TFomQVNcT91MR/7Y8LemNLmhUJrFLq8UADwO2FwYqSScZLJUhwn4QVCHHUS0BIn3lFGRbiCykg5lbE4FSXpYZA2xhA/cGmqdR0ZMz1M9GD7I8B9qtotIhcDdwOni8gi4Gig2iv3lIi8R1Wfxe3W2ikiJbhB8ingnswdi8hFwEUACxYsGFXl1m/ZRWNbF2e9c4zmPFaF7hZor4OOBuioQ9vrSO76G11v/oWut3bT9bd6una1k2zLPtCsPiUVUZJhpSuqdJT6aD3YR1PMT33MR10M9saUupj2X4mUHgqooE6MVNILg/5AcEPB1xEn3lvOrFA5FeFKZkcrqCyPDLjKqC8Q+i5NLbOuI2NmtGIGyU5gftpyNQcG1QFQ1fq0xduAH3i//zdgvaq2AYjI48ApwLOqutPbtlVE7sXtQhsUJKq6FlgLUFNTM6oWS+e/nMdcaUWPfzF7d4qThM4GLxjqoaNuQEgcWF9Pd3sdrZ2NtHYoHY1BehuCaEOQQL2fQKf7KAnF7T7atyDAzoPC7E0I+6Ip9sYcGqOpwaEAqPoPBEF/ILi/B3tLiGs5c3rLqYxUUBWtZHasnIp4hMq0rqP+gLCuI2PMKBQzSDYAi0XkcNwAOQ/4h/QCIjJXVXd7i+cAfd1XfwMuFJHv43ZtvQ/4sYgEgHJVrRORIPAh4PfFOoCrD2+i3a8cd/8F/GNJkMOa99He1UhbTzNt3S20OV20+YQ2n4828bk/+5b9AUKtfqr2wSF7lAV7/SzcU0Vpp3sZWlCgdhZsXyS8dZCwfa7w9qwAnYEoOBE0FUGdmBsOqThhLSXeU85cLaM8XEFlpJLZsSoOSpT1X3nU14VUGXdbCZGgdR0ZY4qvaEGiqkkRuQR4Evfy3ztU9TURWQNsVNV1wKUicg7uvGQNwGe8zR8ETgf+jPtF/QlVfURE4sCTXoj4cUPk1mIdQ1B8gMPWrj/x1R7vAXQB7xWLATEA/ArVjSEW1vo5dA8cusfh0L3dJLrdx2gkRfhbRQl/qq5kW8Vs/lJ5EHVz5hNLVFIZKWNWvJSliXLel4hRlQhTlQhRFQ/1B4R1HRljJjOZhuPUg9TU1OjGjSO/Wvhvv/k53a89wluHv48rtgodoSTaG+KQpi4WNTRwREMdRzTu44im3cSTXQAkfQHqDppPS/VCuhYuQZYcSfTII6kqT/SHREUsZMFgjJn0RGSTqtYMVW6iB9snt8c3EaiP8IFvfo2SW+8l+W+/omL32/i73dDQUIjA4iXE3v8REscsI7JsGeEjjkBCoYmttzHGjCMLkjwkGEJ7e2l+9LeU33AN4UVHEPv4/yCydCmRpUsJH7EQCdg/oTFmZrOzYB6Bqiran32WXVddRaymhvm3rsUXiUx0tYwxZlKxKczyCB5yCNrTQ+TII6n+2c0WIsYYk4W1SPJIrFhB97ZtHPSNr+NPJCa6OsYYMylZkOQROXIJ867/4URXwxhjJjXr2jLGGFMQCxJjjDEFsSAxxhhTEAsSY4wxBbEgMcYYUxALEmOMMQWxIDHGGFMQCxJjjDEFmRGPkReR/cBfR7n5LKBuDKszFdgxzwx2zNNfocd7qKrOHqrQjAiSQojIxuE8j386sWOeGeyYp7/xOl7r2jLGGFMQCxJjjDEFsSAZ2tqJrsAEsGOeGeyYp79xOV4bIzHGGFMQa5EYY4wpiAVJHiKyUkTeFJFtInLVRNdnrInIfBF5WkQ2i8hrInKZt75SRJ4Ska3ez4qJrutYExG/iLwkIo96y4eLyJ+8z/oBEQlNdB3HkoiUi8iDIvKGiLwuIqdM989ZRC73/rt+VUTuE5HIdPucReQOEdknIq+mrcv6uYrrRu/YXxGRd45VPSxIchARP3ATcBawFDhfRJZObK3GXBL4X6q6FFgOfMk7xquAf1fVxcC/e8vTzWXA62nL1wE/UtVFQCNwwYTUqnh+AjyhqkcBf4d77NP2cxaRecClQI2qvgPwA+cx/T7nu4CVGetyfa5nAYu910XAz8aqEhYkuZ0EbFPV7araA9wPnDvBdRpTqrpbVV/0fm/FPbnMwz3Ou71idwMfmZgaFoeIVANnA7d5ywKcDjzoFZlWxywiZcB7gdsBVLVHVZuY5p8z7gywUREJADFgN9Psc1bVPwINGatzfa7nAveoaz1QLiJzx6IeFiS5zQN2pC3XeuumJRE5DDge+BNwkKru9t7aAxw0QdUqlh8DVwApb7kKaFLVpLc83T7rw4H9wJ1ed95tIhJnGn/OqroTuB74G26ANAObmN6fc59cn2vRzmkWJAYRSQD/BnxZVVvS31P3sr5pc2mfiHwI2Keqmya6LuMoALwT+JmqHg+0k9GNNQ0/5wrcb+CHA4cAcQZ3AU174/W5WpDkthOYn7Zc7a2bVkQkiBsiv1DVX3ur9/Y1eb2f+yaqfkVwKnCOiLyN2115Ou74QbnXBQLT77OuBWpV9U/e8oO4wTKdP+e/B95S1f2q2gv8Gvezn86fc59cn2vRzmkWJLltABZ7V3mEcAfq1k1wncaUNzZwO/C6qt6Q9tY6YJX3+yrg4fGuW7Go6tWqWq2qh+F+pn9Q1U8CTwMf84pNt2PeA+wQkSO9Ve8HNjONP2fcLq3lIhLz/jvvO+Zp+zmnyfW5rgM+7V29tRxoTusCK4jdkJiHiHwQtz/dD9yhqt+d4CqNKRF5N/As8GcOjBd8HXec5JfAAtynJn9cVTMH9KY8ETkN+KqqfkhEFuK2UCqBl4B/VNXuiazfWBKR43AvLggB24HP4n6RnLafs4j8b+ATuFcnvgR8DndMYNp8ziJyH3Aa7lN+9wL/DDxEls/VC9Sf4nbxdQCfVdWNY1IPCxJjjDGFsK4tY4wxBbEgMcYYUxALEmOMMQWxIDHGGFMQCxJjjDEFsSAxxhhTEAsSYyYJETmt77H2hZQxZrxZkBgzTN4dwfb/jDEZ7H8KY/IQkcO8yc3uAV4FPiUiz4vIiyLyK++Bl4jIB71JozZ5kwflbDWIyEnePl4Skf9Me3RJepnVIvIvXrmtInJh2tuJtEmqfuHdsYyIfFtENngTOa3tW29MsVmQGDO0xcDNwPtwJ0L6e1V9J7AR+IqIRICfA2ep6gnA7CH29wbwHu9JvN8Gvpej3LG4D5U8Bfi2iBzirT8e+DLuhGsLcR9GCPBTVT3Rm8gpCnxoxEdqzCgEhi5izIz3V1Vd7z2CfinwnPdlPwQ8DxwFbFfVt7zy9+HOQJdLGXC3iCzGfcR3MEe5h1W1E+gUkadxJ1trAl5Q1VoAEXkZOAz4f8AKEbkCdxKnSuA14JHRHbIxw2dBYszQ2r2fAjylquenv+k9EHEkvgM8rar/zZtQ7Jkc5TIfhNe3nP6QQQcIeK2im3Gnlt0hIquByAjrZcyoWNeWMcO3HjhVRBYBiEhcRJYAbwILvVAA94mz+ZRxYB6Iz+Qpd66IRESkCvcJrxvylO0LjTpv3OZjecoaM6YsSIwZJlXdj3viv09EXsHr1vK6n74IPCEim4BW3Kldc/kB8H0ReYn8vQKv4M6fsR74jqruylO3JuBW3AsCniR/6Bgzpuwx8saMARFJqGqbd6XUTcBWVf1RAftbDbSp6vVjVUdjisVaJMaMjQu9ge/XcLuufj7B9TFm3FiLxJgiEZHPApdlrH5OVb80EfUxplgsSIwxxhTEuraMMcYUxILEGGNMQSxIjDHGFMSCxBhjTEEsSIwxxhTk/wN1T+pWTMNqHwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 0.1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取调优后的正则参数\n",
    "# neg_log_loss 得分越大，模型越好\n",
    "# 上一步得到的 neg_log_loss = -0.588041\n",
    "#            xgb2_2.reg_alpha = 0,xgb2_2.reg_lambda = 1\n",
    "if gsearch5_1.best_score_> -0.588041:\n",
    "    reg_alpha1=gsearch5_1.best_params_['reg_alpha']\n",
    "    reg_lambda1=gsearch5_1.best_params_['reg_lambda']\n",
    "else:\n",
    "    reg_alpha1=0\n",
    "    reg_lambda1=1\n",
    "\n",
    "reg_alpha1,reg_lambda1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0, 2, 3], 'reg_lambda': [0.0, 0.2, 0.30000000000000004]}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 然后设置比较小的正则步长\n",
    "\n",
    "if reg_alpha1>1:\n",
    "    # 大于1的数，细步长取1\n",
    "    reg_alpha_List=[reg_alpha1+1*i for i in range(-2,3,1)]\n",
    "elif reg_alpha1<=0.00001:\n",
    "    reg_alpha1=0\n",
    "    # 小于1e-5，认为是0，步长取0.0001\n",
    "    reg_alpha_List=[0+0.0001*i for i in range(0,4,1)]\n",
    "else:\n",
    "    # 其他情况步长取本值相同步长\n",
    "    reg_alpha_List=[reg_alpha1+reg_alpha1*i for i in range(-2,3,1)]\n",
    "\n",
    "if reg_lambda1>1:\n",
    "    # 大于1的数，细步长取1\n",
    "    reg_lambda_List=[reg_lambda1+1*i for i in range(-2,3,1)]\n",
    "elif reg_lambda1<=0.00001:\n",
    "    reg_lambda1=0\n",
    "    # 小于1e-5，认为是0，步长取0.0001\n",
    "    reg_lambda_List=[0+0.0001*i for i in range(0,4,1)]\n",
    "else:\n",
    "    # 其他情况步长取本值相同步长\n",
    "    reg_lambda_List=[reg_lambda1+reg_lambda1*i for i in range(-2,3,1)]\n",
    "\n",
    "# 已经计算过的参数，不再重复计算\n",
    "if reg_alpha1 in reg_alpha_List:\n",
    "    reg_alpha_List.remove(reg_alpha1)\n",
    "    \n",
    "if reg_lambda1 in reg_lambda_List:\n",
    "    reg_lambda_List.remove(reg_lambda1)\n",
    "\n",
    "# 删除负数\n",
    "for i in reg_alpha_List:\n",
    "    if i<0:\n",
    "        reg_alpha_List.remove(i)\n",
    "\n",
    "for i in reg_lambda_List:\n",
    "    if i<0:\n",
    "        reg_lambda_List.remove(i)\n",
    "        \n",
    "# 定义grid\n",
    "param_test5_2 = {\n",
    "        'reg_alpha':reg_alpha_List, \n",
    "        'reg_lambda':reg_lambda_List  \n",
    "} \n",
    "param_test5_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=5, missing=None, n_estimators=267,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.3),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_alpha': [0, 2, 3], 'reg_lambda': [0.0, 0.2, 0.30000000000000004]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=267,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch5_2 = GridSearchCV(xgb5_2, param_grid = param_test5_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_2.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.587763 using {'reg_alpha': 2, 'reg_lambda': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_2.best_score_, gsearch5_2.best_params_))\n",
    "test_means = gsearch5_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_2.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 0.0)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取调优后的正则参数\n",
    "# neg_log_loss得分越大，模型越好\n",
    "\n",
    "#b步得到的neg_log_loss = -0.588041\n",
    "if gsearch5_2.best_score_> gsearch5_1.best_score_:\n",
    "    reg_alpha_bst=gsearch5_2.best_params_['reg_alpha']\n",
    "    reg_lambda_bst=gsearch5_2.best_params_['reg_lambda']\n",
    "else:\n",
    "    reg_alpha_bst=gsearch5_1.best_params_['reg_alpha']\n",
    "    reg_lambda_bst=gsearch5_1.best_params_['reg_lambda']\n",
    "\n",
    "if gsearch5_2.best_score_> -0.588041:\n",
    "    reg_alpha_bst=gsearch5_2.best_params_['reg_alpha']\n",
    "    reg_lambda_bst=gsearch5_2.best_params_['reg_lambda']\n",
    "else:\n",
    "    reg_alpha_bst=0\n",
    "    reg_lambda_bst=1\n",
    "    \n",
    "reg_alpha_bst,reg_lambda_bst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
